<?php
/**
 * @package		简易CMS
 * @subpackage	自动采集处理类
 * @version		$Id: class_focus.php 25 2012-08-29 17:55:19Z htmambo@gmail.com $
 * @author		Hoping
 * @copyright	Copyright (C) 2010 Hoping Software Studio.
 */

if(!defined('IN_JYCMS')) {
	exit('Access Denied');
}

/**
 * 搜索引擎采集，列表入库
 *
 */
class focus {

	var $errorstr = array();
	var $keywords = "";
	var $threadList = array();
	var $groupID = 0;
	var $groupInfo = array();
	var $lastReadTime = 0;
	var $url = "";
	/**
	 * 刷新频率，默认每天读取一次
	 *
	 * @var 整数
	 */
	var $frequency = 86400;
	var $charset = 'UTF-8';
	var $rule = '';
	var $replace = '';
	var $format = 'rss';
	var $thl = '';
	var $chl = '';
	var $number = 10;
	var $state;
	var $displaystatus = false;
	var $snoopy;
	var $fullcontent = 0;
	function focus() {
		global $snoopy;
		if (!$snoopy) {
			require_once libfile('class/snoopy');
			$snoopy = new Snoopy();
		}
		$this->snoopy = & $snoopy;
		$this->snoopy->rawheaders["Pragma"] = "no-cache";
		$this->snoopy->maxredirs = 2;
		$this->snoopy->offsiteok = false;
		$this->snoopy->expandlinks = false;
	}

	function setfreguency($frequency = 86400) {
		$this->frequency = $frequency;
	}

	function readthreadlist($keywords = '') {
		global $_G;
		if (!$this->url)
			$this->url = $this->rssURL;
		if (!$this->url) {
			loadcache('engine');
			$engine = $_G['cache']['engine'];
			if (!$engine) {
				$this->seterror("对不起，系统中尚无可用的搜索引擎！");
				return false;
			}
			foreach ($engine as $v) {
				$blogurl[] = $v;
			}
			$count = count($blogurl);
			if ($count > 1) {
				$i = rand(0, $count - 1);
			} else {
				$i = 0;
			}
			$this->charset = $blogurl[$i]['charset'];
			$this->url = trim($blogurl[$i]['url']);
			$this->replace = $blogurl[$i]['replace'];
			$this->rule = $blogurl[$i]['rule'];
			$this->format = $blogurl[$i]['format'];
			$this->fullcontent = $blogurl[$i]['fullcontent'];
			$thl = trim($blogurl[$i]['titlehl']);
			if ($thl) {
				$thl = processrule($thl);
				$thl = str_ireplace("{keyword}", "([^']" . '[^"]' . "+)", $thl);
			}
			$chl = trim($blogurl[$i]['contenthl']);
			if ($chl) {
				$chl = processrule($chl);
				$chl = str_ireplace("{keyword}", "([^']" . '[^"]' . "+)", $chl);
			}
			$this->thl = $thl;
			$this->chl = $chl;
		}
		if (!$keywords && !$this->keywords) {
			$this->seterror("对不起，您没有设置要检索的关键词！");
			return false;
		}
		if (!$keywords) {
			$keywords = $this->keywords;
		} else {
			$this->keywords = $keywords;
		}
		if ($this->charset != CHARSET) {
			$keywords = diconv($keywords, CHARSET, $this->charset);
		}
		if ($this->onlyMatchTitle == true) {
			$keywords = "title:" . rawurlencode($keywords);
		} else {
			$keywords = rawurlencode($keywords);
		}
		$this->url = str_ireplace('{keyword}', $keywords, $this->url);
		$this->url = str_ireplace('{number}', $this->number, $this->url);
		$this->state[] = "加载关键词“" . $this->keywords . "”的信息列表";
		if ($this->format == 'rss') {
			$rss = $this->readrss($this->url, false);
			if (!$rss)
				return false;
			$thread_list = (array) $rss['items'];
			if (0 <= sizeof($thread_list) && !empty($start_list)) {
				foreach ((array) $start_list as $start) {
					$rss = $this->readrss($this->url . $start, false);
					$_thread_list = (array) $rss['items'];
					$thread_list = array_merge($thread_list, $_thread_list);
					if (!(sizeof($_thread_list) < 1)) {
						continue;
					}
					break;
				}
			}
			$thread_len_list = array();
			foreach ($thread_list as $key => $thread) {
				if (!$thread['content'] && $thread['description'])
					$thread_list[$key]['content'] = $thread['description'];
				$thread_len_list[$key] = strlen($thread['description']);
			}
			array_multisort($thread_len_list, SORT_DESC, $thread_list);
		} else {
			$this->processrule();
			//分析采集规则
			$this->rule = processrule($this->rule);
			preg_match_all('/\{([a-zA-Z]+)\}/i', $this->rule, $matches);
			if ($matches[1]) {
				$i = 1;
				foreach ($matches[1] as $k => $v) {
					if ($v !== 'ignore') {
						$rules[$i] = $v;
						if ($v !== 'content') { //除内容字段外，其它字符均不得包含引号与双引号
							$matches[1][$k] = "([^'][^>]" . '[^"]' . "+)";
						} else {
							$matches[1][$k] = "(.*?)";
						}
						$i++;
					} else {
						$matches[1][$k] = ".*?";
					}
				}
			}
			$this->rule = str_ireplace($matches[0], $matches[1], $this->rule);

			//读取HTTP页面
			$sourcehtml = $this->geturlcontent($this->url, $this->charset);
			if (!$sourcehtml) {
				return false;
			}
			//替换掉<body>....</body>以外的内容
			preg_match("/<body(.*)<\/body>/isU", $sourcehtml, $matches);
			if ($matches[1])
				$sourcehtml = "<body" . $matches[1] . "</body>";
			$sourcehtml = $this->processrepl($sourcehtml);
			preg_match_all("~" . $this->rule . "~iU", $sourcehtml, $matches);
			unset($matches[0]);
			$thread_list = array();
			if ($matches[1]) {
				//处理采集结果
				foreach ($matches as $k => $v) {
					if ($rules[$k]) {
						if (is_array($v)) {
							foreach ($v as $j => $i) {
								$i = trim($i);
								if ($rules[$k] == 'title') {
									$i = clearhtml($i);
									if ($thl) {
										$i = preg_replace('/' . $thl . '/i', "\\1", $i);
									}
								}
								$v[$j] = $i;
							}
							$thread_list[$rules[$k]] = $v;
						}
					}
				}
			}
		}
		$this->threadList = $thread_list;
	}

	function readrss($url, $cache = false) {
		require_once libfile('class/rss');
		$lr = new lastRSS();
		$items = $lr->Get($url);
		return $items;
	}

	function groupinfo($id) {
		$sql = "SELECT * from `#__focus_keywords`" . (" where id=" . $id);
		$query = DB::query($sql);
		return DB::fetch_array($query);
	}

	function read($id, $keywords = "", $refresh = 0, $number = -1) {
		global $control;
		$this->groupInfo = is_array($id) ? $id : $this->groupinfo($id);
		$this->groupID = $this->groupInfo['id'];
		$this->keywords = trim($this->groupInfo['name']);
		$fn = "catid";
		if (!$this->groupInfo[$fn]) {
			$sql = "SELECT catid FROM #__common_category WHERE catname='" . $this->keywords . "'";
			$catid = DB::result_first($sql);
			$data = array(
				'allowcomment' => 1,
				'catname' => $this->keywords
			);
			if (!$catid) {
				$catid = DB::insert('common_category', $data, true);
				require_once libfile('function/cache');
				updatecache('portalcategory');
			}
			if (!$catid) {
				$this->seterror("没有指定关键词“" . $this->keywords . "”的分类! 系统拒绝读取！");
				return false;
			}
			$data = array('catid' => $catid);
			DB::update('focus_keywords', $data, "id='" . $this->groupID . "'");
			$this->groupInfo[$fn] = $catid;
		}
		$this->state[] = "开始加载关键词“" . $this->keywords . "”的信息列表";
		if (!$this->groupInfo) {
			$this->seterror("没有指定关键词!");
			return false;
		}
		$fn = "readtime";
		$this->lastReadTime = $refresh == 0 ? $this->groupInfo[$fn] : $refresh;
		$timestamp = time();
		if ($timestamp - $this->frequency < $this->lastReadTime) {
			$this->seterror("关键词“" . $this->keywords . "”不需要读取!");
			return false;
		}
		$this->state[] = "开始处理搜索引擎";

		$this->readthreadlist($this->keywords);
		$this->state[] = "关键词“" . $this->keywords . "”的信息列表处理结束，共读取到" . count($this->threadList) . "条信息";
		if (!count($this->threadList)) {
			$this->seterror("没有读取到关键词“" . $this->keywords . "”的内容!");
			$this->updatereadcount();
			return false;
		}
		$this->filter($number, 1);
		$this->state[] = "关键词“" . $this->keywords . "”的信息过滤结束";

		//pre($this->groupInfo);
		if (!count($this->threadList)) {
			$this->state[] = "没有读取到关键词“" . $this->keywords . "”的信息列表";
			$this->updatereadcount();
			return false;
		}
		$total = 0;
		$i = 1;
		foreach ($this->threadList as $data) {
			if ($number != -1 && $number < $i) {
				break;
			}
			$id = $this->insertthread($data);
			if ($id)
				$total++;
			++$i;
		}
		if (!$total) {
			$this->seterror("没有读取到关键词“" . $this->keywords . "”的内容!");
		}
		$this->updatereadcount($total);
		$this->state[] = "关键词“" . $this->keywords . "”的信息处理完成";
		return $total;
	}

	function updatereadcount($total = 0) {
		//更新最后检索时间
		$frequency = time();
		$fn = "readtime";
		$sql = "UPDATE `#__focus_keywords`" . (" set " . $fn . "=" . $frequency . " where id={$this->groupID}");
		$query = DB::query($sql, "SKIP_ERROR");
		//写入检索记录
		$today = date("Y-m-d H:i:s", TIMESTAMP);
		$note = serialize($this->errorstr);
		$data = array(
			'gid' => $this->groupID,
			'nums' => $total,
			'keyword' => $this->keywords,
			'dateline' => $today,
			'note' => $note
		);
		DB::Insert('focus_log', $data);

		if ($total) {
			$this->seterror("添加" . $total . "条");
		}
	}

	function insertthread($data) {
		//写入采集回来的数据
		global $_G;
		if(!$_G['ch']) {
			require_once libfile('class/chinese');
		}
		$_G['ch'] = new Chinese('UTF-8', 'PINYIN');
		$fn = "catid";
		$row = array();
		$row['catid'] = $this->groupInfo['catid'];
		$row['kid'] = $this->groupInfo['id'];
		$row['keyword'] = $this->groupInfo['name'];
		$row['title'] = trim(dhtmlspecialchars(clearhtml(htmlspecialchars_decode($data['title']))));
		$row['title'] = preg_replace("/font color=\#.{6}(.*?)\/font/", "\\1", $row['title']);
		$row['summary'] = strip_tags($data['content']);
		$row['fromrss'] = 1;
		$row['from'] = '网络文摘';
		$row['fromurl'] = $data['link'];
		$row['author'] = $data['author'];

		$iconfig = unserialize($_G['setting']['importconfig']);

		$uid = $iconfig['authorid'];
		$username = $iconfig['author'];
		if (!$uid)
			$uid = 1;
		if (!$username)
			$username = '网文转摘';

		if (!$row['author']) {
			$row['author'] = $username;
		}
		$row['uid'] = $uid;
		$row['readed'] = $this->fullcontent;

		$article = array();
		$v = array('catid', 'title', 'author', 'from', 'fromurl', 'summary', 'kid', 'uid', 'fromrss');
		if (!$row ['uid'])
			$row ['uid'] = 1;
		$article ['username'] = $row ['author'];
		$article ['dateline'] = TIMESTAMP;
		$article ['allowcomment'] = $data ['contents'] = 1;
		foreach ($v as $key) {
			$article [$key] = $row [$key];
		}
		$article ['summary'] = daddslashes(cutstr(strip_tags($article ['summary']), 200));
		$aid = DB::Insert('portal_article_title', $article, 1);
		DB::query("UPDATE #__common_category SET articles=articles+1 WHERE catid = '$article[catid]'");

		if ($this->fullcontent) {
			$msg = '“全文收录”';
			$article = array();
			$article['aid'] = $aid;
			$article['content'] = daddslashes($data['content']);
			if ($iconfig ['saveimg']) {
				$article['content'] = processpic($article, $data['link']);
			}
			$article['pageorder'] = 1;
			$article['dateline'] = TIMESTAMP;
			DB::Insert('portal_article_content', $article);
		}
		if (!$aid) {
			$this->seterror("添加《" . $data['title'] . "》时出错！");
		} else {
			$itemurl = "index.php?task=view&aid=" . $aid;
			$this->seterror("已经添加《<a href=" . $itemurl . " target=_blank>" . $data['title'] . "</a>》" . $msg);
			$this->writehistory($data['link']);
		}
		return $aid;
	}

	//对指定的URL只在系统中采集一次
	function single() {
		$tt = count($this->threadList);
		foreach ($this->threadList as $k => $v) {
			$url_hash = md5($v['link']);
			$sql = "select id from #__focus_urlhistory where url = '" . trim($v['link']) . "' or url_hash='" . $url_hash . "'";
			$query = DB::query($sql, "SKIP_ERROR");
			if ($row = DB::fetch_array($query)) {
				unset($this->threadList[$k]);
			}
		}
		$tt = $tt - count($this->threadList);
		$this->state[] = "检查关键词“" . $this->keywords . "”的重复信息[共重复 $tt 条，还余" . count($this->threadList) . "条]";
	}

	function filter($num = -1, $sysfilter = 1) {
		$url_hashs = "";
		foreach ($this->threadList as $key => $thread) {
			$i = $thread['title'];
			$i = trim(clearhtml(htmlspecialchars_decode($i)));
			$i = preg_replace('/font color=#.{6}/isU', '', $i);
			$i = preg_replace('/\/font/isU', '', $i);
			$i = str_replace('- ', '-', $i);
			$i = str_replace(' -', '-', $i);
			//清除标题中的一些常见的垃圾信息
			$regex[] = '/\-淘宝网\-[^\-]+\-淘宝空间.*/i';
			$regex[] = '/\-[^\-]+\-搜狐博客.*/i';
			$regex[] = '/\-[^\-]+\-[^\-]+\-和讯博客.*/i';
			$regex[] = '/\-[^\-]+\-网易博客.*?/i';
			$regex[] = '/_.*?/i';
			$regex[] = '/powered by.*?/iU';
			$i = preg_replace($regex, '', $i);
			$this->threadList[$key]['title'] = $i;
			$url_hash = md5($thread['link']);
			$url_hashs .= $comma . "'" . $url_hash . "'";
			$this->threadList[$key]['url_hash'] = $url_hash;
			$comma = ",";
		}
		$this->single();
		$thread_list = array();
		$thread_title_list = array();
		$count = 0;

		$lists = array("title" => "标题", 'author' => "作者", "content" => "正文", "link" => "原文链接");
		if ($this->rule) {
			$tmp = explode("\n", $this->rule);
			foreach ($tmp as $k => $v) {
				$v = strtolower($v);
				$v = explode("=", $v);
				$rules[trim($v[0])] = trim($v[1]);
			}
		}
		foreach ($lists as $k => $v) {
			if (!$rules[$k])
				$rules[$k] = $k;
		}
		foreach ($this->threadList as $key => $thread) {
			if ($num != -1 && $num <= $count) {
				break;
			}
			$thread['title'] = strip_tags($thread['title']);
			if (!isset($thread_title_list[$thread['title']])) {
				$thread_title_list[$thread['title']] = 1;
				$arr = array();
				foreach ($lists as $j => $v) {
					if ($thread[$rules[$j]]) {
						$k = $rules[$j];
						$arr[$j] = $thread[$k];
					}
				}
				//pre($arr);echo $this->charset;
				$reg = '/&amp;([a-z]+);/';
				$arr['title'] = diconv($arr['title'], $this->charset);
				$arr['content'] = diconv(str_ireplace(array("&nbsp;", "\n", "\r\n", "\\s"), "", $arr['content']), $this->charset);

				$arr['title'] = htmlspecialchars_decode($arr['title']);
				$arr['content'] = htmlspecialchars_decode($arr['content']);
				$arr['title'] = preg_replace('/<font[^>]+>(.+?)<\/font>/', '\\1', $arr['title']);
				$arr['title'] = preg_replace('/font color=#[0-9a-fA-F]+(.+?)\/font/', '\\1', $arr['title']);
				$arr['content'] = preg_replace('/<font[^>]+>(.+?)<\/font>/isU', '\\1', $arr['content']);
				$arr['content'] = preg_replace('/font color=#[0-9a-fA-F]+(.+?)\/font/isU', '\\1', $arr['content']);


				$arr['title'] = preg_replace($reg, "&\\1", $arr['title']);
				$arr['title'] = clearhtml($arr['title']);
				if ($arr['title'] && $this->thl) {
					$arr['title'] = preg_replace('/' . $this->thl . '/i', "\\1", $arr['title']);
				}
				$arr['content'] = preg_replace($reg, "&\\1", $arr['content']);
				if ($arr['content'] && $this->chl) {
					$arr['content'] = preg_replace('/' . $this->chl . '/i', "\\1", $arr['content']);
				}
				$arr['content'] = getrobotmessage($arr['content'], $arr['link'], 1, 0);
				if (!$arr['author'] && $thread['dc']['creator'])
					$arr['author'] = $thread['dc']['creator'];
				$arr['author'] = diconv($arr['author'], $this->charset, CHARSET);
				//$row = getrule($arr['link']);
				if ($sysfilter) {
					$err = filter($arr['title'], "title", false, $statistic);
					if (!$err) {
						$cc = "newscontent";
						$err = filter($arr['content'], $cc, false, $statistic);
						if (!$err) {
							$arr['statistic'] = $statistic;
							$thread_list[] = $arr;
							++$count;
						} else {
							$this->seterror($err);
						}
					} else {
						$this->seterror($err);
					}
				} else {
					$thread_list[] = $arr;
				}
				//$thread_list[] = $arr;
			}
		}
		$this->threadList = $thread_list;
	}

	function seterror($errstr) {
		if ($this->displaystatus)
			echo $errstr . "<br>";
		$this->errorstr[] = $errstr;
	}

	function geturlcontent($url, $charset) {
		if ($this->snoopy->fetch($url)) {
			$text = $this->snoopy->results;
			if (!$charset || $charset == 'auto') {
				$charset = $this->snoopy->htmlheader['charset'];
			}
			if (!$charset)
				$charset = "gb2312";
			$text = diconv($text, $charset, CHARSET);
			return $text;
		} else {
			trigger_error("读取 $url 时出错！");
			return false;
		}
	}

	function processrule() {
		$replace = trim($this->replace);
		$this->replace = array();
		if ($replace) {
			$replace = explode("<==newline==>", $replace);
			$replaces = array();
			foreach ($replace as $v) {
				$v = explode("<==>", $v);
				//				if(count($v)==1){
				$v[0] = trim($v[0]);
				//				}
				$v[1] = trim($v[1]);
				//				if($v[1]=='BR') {
				//					$v[1]='\n';
				//				}
				$this->replace[] = array("find" => $v[0], "replace" => $v[1]);
			}
		}
	}

	function processrepl($html = '') {
		if ($this->replace) {
			foreach ($this->replace as $v) {
				if ($v['replace'] == 'BR')
					$v['replace'] = "\n";
				$v['replace'] = str_replace('/', '\\/', $v['replace']);
				$v['find'] = str_replace('/', '\\/', $v['find']);
				$html = preg_replace("/" . $v['find'] . "/isU", $v['replace'], $html);
			}
		}
		return $html;
	}

	function writehistory($url) {
		$url_hash = md5($url);
		$sql = "REPLACE INTO #__focus_urlhistory (url_hash,url,dateline) values ('$url_hash','" . $url . "','" . time() . "')";
		$query = DB::query($sql, "SKIP_ERROR");
	}

}
